# coding:utf-8
import scrapy
from scrapy.selector import Selector
from scrapy.spiders import Spider
from ..items import NewsItem
from ..spiders import utils_crawler

source = u'财新网'


class CaiXinSpider(Spider):
    name = "caixin"
    start_urls = [
        "http://economy.caixin.com/",
        "http://finance.caixin.com/",
        "http://companies.caixin.com/"
    ]

    def parse(self, response):
        item_urls = []
        sel = Selector(response)
        hrefs = sel.xpath('//div[@id="listArticle"]//a/@href').extract()
        for href in hrefs:
            if (href.find("http://economy.caixin.com/2") != -1) or (href.find("http://finance.caixin.com/2") != -1) \
                    or (href.find("http://companies.caixin.com/2") != -1):
                item_urls.append(href)

        print(item_urls)
        for item_url in item_urls:
            yield scrapy.Request(url=item_url, callback=self.parse_details)

    def parse_details(self, response):
        item = NewsItem()
        sel = Selector(response)

        item['title'] = sel.xpath('//div[@id="conTit"]/h1/text()').extract()[0].strip()
        item['href'] = response.url
        item['time'] = sel.xpath('//div[@id="artInfo"]/text()').extract()[0].encode('utf-8')\
                           .replace('来源于', '').strip().replace('年', '-').replace('月', '-').replace('日', '') + ':00'
        item['content'] = utils_crawler.deal_content(sel.xpath('//div[@id="Main_Content_Val"]//p/text()').extract())
        item['source'] = source
        item['classify'] = sel.xpath('//div[@class="crumb"]/span/a/text()').extract()[0]
        item['image_urls'] = sel.xpath('//div[@id="Main_Content_Val"]//img/@src').extract()

        return item
